In [1]:
import os
import cv2
import matplotlib
import numpy as np
from matplotlib import pyplot as plt
#from google.colab.patches import cv2_imshow

%matplotlib inline

Read and resize data

In [2]:
path_original="./office/" 
path_resized="./resized/"
resize_divide=4

images_original = [cv2.imread(f"{path_original}{image}", cv2.IMREAD_COLOR) for image in os.listdir(f'{path_original}')]

height, width, _ = images_original[0].shape
height = height // resize_divide
width = width // resize_divide

images = [cv2.resize(image, (width, height)) for image in images_original]

for i, image in enumerate(images):
        cv2.imwrite(f"{path_resized}img{i}.png", image)
        
images = [cv2.imread(f"{path_resized}{image}", cv2.IMREAD_COLOR) for image in os.listdir(f'{path_resized}')]      

Setup

In [3]:
# termination criteria
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)

# prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,8,0)
objp = np.zeros((7 * 9, 3), np.float32)
objp[:,:2] = np.mgrid[0:7,0:9].T.reshape(-1,2)

print(objp[:6])
print(".......")
print(objp[56:])
[[0. 0. 0.]
 [1. 0. 0.]
 [2. 0. 0.]
 [3. 0. 0.]
 [4. 0. 0.]
 [5. 0. 0.]]
.......
[[0. 8. 0.]
 [1. 8. 0.]
 [2. 8. 0.]
 [3. 8. 0.]
 [4. 8. 0.]
 [5. 8. 0.]
 [6. 8. 0.]]
In [4]:
# Arrays to store object points and image points from all the images.
objpoints = [] # 3d point in real world space
imgpoints = [] # 2d points in image plane.

cv2.findChessboardCorners() : finds pattern in chess board. In my case 7x9 grid. It returns the corner points and retval which will be True if pattern is obtained. These corners will be placed in an order (from left-to-right, top-to-bottom);

cv2.cornerSubPix() : increase corner's accuracy;

cv2.drawChessboardCorners() : draws the pattern;

In [5]:
for img in images:
    
    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

    # Find the chess board corners
    ret, corners = cv2.findChessboardCorners(gray, (7,9),None)

    # If found, add object points, image points (after refining them)
    if ret == True:
        objpoints.append(objp)

        corners2 = cv2.cornerSubPix(gray,corners,(11,11),(-1,-1),criteria)
        imgpoints.append(corners2)

        # Draw and display the corners
        img = cv2.drawChessboardCorners(img, (7,9), corners2,ret)
        plt.figure(figsize=(8,16))
        plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
        plt.show()
   

Calibration

cv2.calibrateCamera() : returns the camera matrix, distortion coefficients, rotation and translation vectors etc.

In [6]:
ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1],None,None)
print("ret : "+str(ret))
print("mtx : "+str(mtx))
print("dist : "+str(dist))
print("rvecs : "+str(rvecs))
print("tvecs : "+str(tvecs))
ret : 0.48162935651571376
mtx : [[875.60429217   0.         507.73506874]
 [  0.         873.09982091 379.41723341]
 [  0.           0.           1.        ]]
dist : [[ 0.13646777 -0.43547317  0.00293569  0.00361166  0.49329303]]
rvecs : [array([[ 0.16725619],
       [ 0.31796462],
       [-1.53938779]]), array([[-0.04447836],
       [ 0.4214119 ],
       [ 1.48871327]]), array([[ 0.34667874],
       [ 0.2482807 ],
       [-1.53409133]]), array([[0.58897805],
       [0.17744005],
       [1.52033984]]), array([[ 0.63110318],
       [-0.14637544],
       [-1.52902218]]), array([[ 0.08419905],
       [ 0.01166781],
       [-1.56402996]]), array([[ 0.33586962],
       [-0.04742262],
       [-1.58958523]]), array([[ 0.05556957],
       [ 0.02948171],
       [-1.57101939]]), array([[ 0.3622207 ],
       [-0.31959964],
       [-1.58834994]]), array([[ 0.25368566],
       [-0.33969248],
       [-1.59639914]]), array([[ 0.08069253],
       [ 0.02693623],
       [-1.56640008]]), array([[0.40773966],
       [0.2440662 ],
       [1.54345201]]), array([[-0.20956436],
       [ 0.06533772],
       [ 1.51249565]]), array([[-0.0678113 ],
       [ 0.56440991],
       [-1.47582068]]), array([[0.13854853],
       [0.25035017],
       [1.56450622]]), array([[ 0.10855574],
       [ 0.10448389],
       [-1.56098947]]), array([[ 0.03855836],
       [-0.05198089],
       [ 1.55890464]]), array([[0.37388433],
       [0.13862496],
       [1.59235054]]), array([[ 0.14392463],
       [-0.1422387 ],
       [-1.54663825]]), array([[ 0.05523866],
       [ 0.01996539],
       [-1.56592928]]), array([[0.01646891],
       [0.35487907],
       [1.51492607]]), array([[ 0.04559373],
       [-0.05648229],
       [ 1.56725224]]), array([[ 0.26041293],
       [ 0.44045604],
       [-1.49433219]]), array([[ 0.26896046],
       [ 0.29069746],
       [-1.54995211]]), array([[0.467969  ],
       [0.12329494],
       [1.53314608]]), array([[ 0.11442806],
       [ 0.04653858],
       [-1.56990364]]), array([[ 0.576553  ],
       [-0.4493936 ],
       [-1.55690387]]), array([[ 0.23334471],
       [-0.03918309],
       [-1.56144024]]), array([[0.08167122],
       [0.31313967],
       [1.53512885]]), array([[-0.0721298 ],
       [ 0.36687701],
       [ 1.46878535]]), array([[ 0.05881258],
       [ 0.0403906 ],
       [-1.57069248]]), array([[0.60409949],
       [0.34263192],
       [1.54856061]]), array([[ 0.56637355],
       [ 0.21290379],
       [-1.50944186]]), array([[ 0.50813491],
       [ 0.17067079],
       [-1.52816544]]), array([[-0.0093008 ],
       [ 0.40000947],
       [-1.53286193]]), array([[ 0.33563261],
       [ 0.15311102],
       [-1.53642848]]), array([[0.1727593 ],
       [0.34011264],
       [1.55980357]]), array([[0.316103  ],
       [0.09405529],
       [1.59626772]]), array([[ 0.21926935],
       [ 0.08166542],
       [-1.54645093]])]
tvecs : [array([[-4.90922025],
       [ 3.49484126],
       [17.63032903]]), array([[ 2.2790326 ],
       [-4.38119626],
       [16.77609204]]), array([[-3.96396477],
       [ 3.42897848],
       [17.42970784]]), array([[ 1.82129212],
       [-2.00815948],
       [13.12794085]]), array([[-2.97845508],
       [ 3.83767383],
       [15.87119916]]), array([[-3.75908934],
       [ 3.08226725],
       [ 9.96515055]]), array([[-2.86871151],
       [ 3.23351663],
       [16.1960491 ]]), array([[-3.89585024],
       [ 2.6709815 ],
       [15.19796808]]), array([[-4.16336102],
       [ 3.0795071 ],
       [13.74592305]]), array([[-4.39551182],
       [ 2.03689942],
       [12.91456611]]), array([[-3.77804046],
       [ 3.01283484],
       [10.23373741]]), array([[ 2.77078994],
       [-3.03147951],
       [16.29876346]]), array([[ 2.83339752],
       [-4.50253229],
       [17.16762964]]), array([[-4.33696122],
       [ 3.52878653],
       [18.84880957]]), array([[ 2.44859365],
       [-2.26465572],
       [16.94082328]]), array([[-4.8014596 ],
       [ 4.04411789],
       [18.85295265]]), array([[ 4.20035781],
       [-4.0360594 ],
       [19.94638924]]), array([[ 2.18257537],
       [-2.45408707],
       [12.98662177]]), array([[-5.58756705],
       [ 2.58096185],
       [15.76946421]]), array([[-3.85873471],
       [ 2.75644057],
       [14.9912611 ]]), array([[ 2.62881644],
       [-3.4982153 ],
       [16.76750605]]), array([[ 4.39442746],
       [-4.24962813],
       [18.02534685]]), array([[-4.27514875],
       [ 4.06321972],
       [19.58437193]]), array([[-4.48121068],
       [ 3.77433843],
       [17.74119911]]), array([[ 2.52449922],
       [-2.66434608],
       [16.26793611]]), array([[-3.90693948],
       [ 2.98653077],
       [10.55531776]]), array([[-2.46322407],
       [ 2.87308961],
       [14.04011381]]), array([[-4.9166479 ],
       [ 2.4918735 ],
       [17.28977235]]), array([[ 2.25964238],
       [-3.66766621],
       [16.51794939]]), array([[ 2.4790025 ],
       [-4.10989821],
       [16.67369861]]), array([[-3.72681017],
       [ 2.49175439],
       [15.50016115]]), array([[ 2.28940176],
       [-2.31157737],
       [12.95702915]]), array([[-4.2622163 ],
       [ 3.83437319],
       [17.51573235]]), array([[-3.61379077],
       [ 4.02136911],
       [16.74500922]]), array([[-6.94116826],
       [ 2.8480128 ],
       [21.77121131]]), array([[-3.30060453],
       [ 4.0224208 ],
       [17.34757571]]), array([[ 2.37009968],
       [-3.11779254],
       [17.1843197 ]]), array([[ 1.80999034],
       [-3.07514426],
       [16.72247922]]), array([[-4.71694268],
       [ 3.07462314],
       [18.71605153]])]

Undistortion

v2.getOptimalNewCameraMatrix() : refines the camera matrix. If the scaling parameter alpha=0, it returns undistorted image with minimum unwanted pixels. So it may even remove some pixels at image corners. If alpha=1, all pixels are retained with some extra black images. It also returns an image ROI which can be used to crop the result.

In [7]:
for image in images:
       
    first = image.copy()
    h,  w, _ = image.shape
    newcameramtx, roi = cv2.getOptimalNewCameraMatrix(mtx, dist, (w,h), 1, (w,h))
        
    undistorted = cv2.undistort(image, mtx, dist, None, newcameramtx)   
    plt.figure(figsize=(20, 40))
    plt.subplot(121)
    plt.title('Original')
    plt.imshow(cv2.cvtColor(first, cv2.COLOR_BGR2RGB))
    plt.axis('off')
    plt.subplot(122)
    plt.title('Undistorted')
    plt.imshow(cv2.cvtColor(undistorted, cv2.COLOR_BGR2RGB))
    plt.axis('off')
    plt.show()
        
In [8]:
path_test_original="./test_original/" 
path_test_resized="./test_resized/"
resize_divide=4

test_original = [cv2.imread(f"{path_test_original}{image}", cv2.IMREAD_COLOR) for image in os.listdir(f'{path_test_original}')]

height, width, _ = test_original[0].shape
height = height // resize_divide
width = width // resize_divide

tests = [cv2.resize(image, (width, height)) for image in test_original]

for i, image in enumerate(tests):
        cv2.imwrite(f"{path_test_resized}test{i}.png", image)
        
images_test = [cv2.imread(f"{path_test_resized}{image}", cv2.IMREAD_COLOR) for image in os.listdir(f'{path_test_resized}')]      

Test on images unseen before

In [9]:
for image in images_test :
       
    first = image.copy()
    h,  w, _ = image.shape
    newcameramtx, roi = cv2.getOptimalNewCameraMatrix(mtx, dist, (w,h), 1, (w,h))
        
    undistorted = cv2.undistort(image, mtx, dist, None, newcameramtx)    
    plt.figure(figsize=(20, 40))
    plt.subplot(121)
    plt.title('Original')
    plt.imshow(cv2.cvtColor(first, cv2.COLOR_BGR2RGB))
    plt.axis('off')
    plt.subplot(122)
    plt.title('Undistorted')
    plt.imshow(cv2.cvtColor(undistorted, cv2.COLOR_BGR2RGB))
    plt.axis('off')
    plt.show()
        
In [10]:
    image=images_test[0]
    first = image.copy()
    h,  w, _ = image.shape
    newcameramtx, roi = cv2.getOptimalNewCameraMatrix(mtx, dist, (w,h), 1, (w,h))
        
    undistorted = cv2.undistort(image, mtx, dist, None, newcameramtx)   
    plt.figure(figsize=(20, 40))
    plt.subplot(121)
    plt.title('Original')
    plt.imshow(cv2.cvtColor(first, cv2.COLOR_BGR2RGB))
    plt.plot([70, 930], [108, 138], 'k-', color='green')
    plt.plot([70, 930], [520, 520], 'k-', color='green')
    plt.plot([250, 250], [20, 720], 'k-', color='green')
    plt.plot([820, 770], [20, 730], 'k-', color='green')
    #plt.axis('off')
    plt.subplot(122)
    plt.title('Undistorted')
    plt.imshow(cv2.cvtColor(undistorted, cv2.COLOR_BGR2RGB))
    plt.plot([70, 930], [108, 138], 'k-', color='green')
    plt.plot([70, 930], [520, 520], 'k-', color='green')
    plt.plot([250, 250], [20, 720], 'k-', color='green')
    plt.plot([820, 770], [20, 730], 'k-', color='green')
    #plt.axis('off')
    plt.show()
In [11]:
    image=images_test[0]
    first = image.copy()
    h,  w, _ = image.shape
    newcameramtx, roi = cv2.getOptimalNewCameraMatrix(mtx, dist, (w,h), 1, (w,h))
        
    undistorted = cv2.undistort(image, mtx, dist, None, newcameramtx)

    #         x, y, w, h = roi
    #         undistorted = undistorted[y:y + h, x:x + w]
        
    plt.figure(figsize=(20, 40))
    plt.subplot(121)
    plt.title('Original')
    plt.imshow(cv2.cvtColor(first, cv2.COLOR_BGR2RGB))
    plt.plot([70, 930], [108, 138], 'k-', color='green')
    plt.plot([70, 930], [520, 520], 'k-', color='green')
    plt.plot([250, 250], [20, 720], 'k-', color='green')
    plt.plot([820, 770], [20, 730], 'k-', color='green')
    #plt.axis('off')
    plt.subplot(122)
    plt.title('Undistorted')
    plt.imshow(cv2.cvtColor(undistorted, cv2.COLOR_BGR2RGB))
    plt.plot([70, 930], [108, 138], 'k-', color='green')
    plt.plot([70, 930], [520, 520], 'k-', color='green')
    plt.plot([250, 250], [20, 720], 'k-', color='green')
    plt.plot([820, 820], [20, 730], 'k-', color='green')
    #plt.axis('off')
    plt.show()
In [12]:
plt.imshow(cv2.cvtColor(undistorted-first, cv2.COLOR_BGR2RGB))
Out[12]:
<matplotlib.image.AxesImage at 0x7f195ae11630>

Re-projection Error

Re-projection error gives a good estimation of just how exact is the found parameters. This should be as close to zero as possible. Given the intrinsic, distortion, rotation and translation matrices, we first transform the object point to image point using cv2.projectPoints(). Then we calculate the absolute norm between what we got with our transformation and the corner finding algorithm. To find the average error we calculate the arithmetical mean of the errors calculate for all the calibration images.

In [13]:
mean_error = 0
for i in range(len(objpoints)):
    imgpoints2, _ = cv2.projectPoints(objpoints[i], rvecs[i], tvecs[i], mtx, dist)
    error = cv2.norm(imgpoints[i],imgpoints2, cv2.NORM_L2)/len(imgpoints2)
    mean_error += error

print ("Re-projection error: "+str( mean_error/len(objpoints)))
Re-projection error: 0.05852943533022806
In [ ]: